#coding: utf-8
from sklearn import tree
import csv
import numpy as np
from sklearn.model_selection import train_test_split       
import random
from sklearn import svm
from sklearn import neighbors, datasets
import ADWIN
from sklearn.cluster import MeanShift, estimate_bandwidth,KMeans
from sklearn import preprocessing
from matplotlib import pyplot


defaultLabel = 1
def splitData(X1,y1):
    ind = set(range(len(y1)))
    lind = random.sample(ind,int(0.2 * len(y1)))
    uind = ind.difference(lind)
    X = [X1[i] for i in lind]
    y = [y1[i] for i in lind]
    U = [X1[i] for i in uind]
    UL = [y1[i] for i in uind]
    return X,y,U,UL

def summary(X,y,nodes,d):
    for i in range(len(nodes)):
        if nodes[i] not in d:
            d[nodes[i]]=[]
        d[nodes[i]].append( (X[i],y[i]) )


def calcConfidence(x,samples):
    numberOfClasses = 2
    p = [0 for i in range(numberOfClasses)]
    for c in range(numberOfClasses):      # two classes
        s = [i for i in samples if i[1]==c]
        K = len(s)
        p[c] = (K+1)*1.0/(len(samples)+numberOfClasses)
    return p

def mypredict(T,U,d):
    nodes = list(T.apply(U))
    result = []
    for i in range(len(U)):
        ui = U[i]
        samples = d[nodes[i]]
        confidence = calcConfidence(ui,samples)
        id = confidence.index(max(confidence))
        result.append([ id,confidence[id] ])
    return result


def selfTrain(X,y,U,UL,maxIt=10000):
    thres = 0.1
    totRemoved = 0

    K = 0.6
    global uTitle
    uTitle = 'take top %.2f%% at each iteration'%(thres*100)
    t = 0
    D = len(X[0])
    print("the U total len: %d\n"%len(U))
    while len(U) > 0 and t < maxIt:
        d= {}   # treenode -> list[ (x,y),... ]
        t = t + 1
        N = len(y)

        m = tree.DecisionTreeClassifier()
        m.fit( np.array(X).reshape(-1,D) ,np.array(y).reshape(-1,1))

        Posmean = np.array([X[i] for i in range(len(y)) if y[i]==1]).mean(0)
        Negmean = np.array([X[i] for i in range(len(y)) if y[i]==0]).mean(0)

        ul = m.predict(np.array(U).reshape(-1,D))

        suspect = []
        tmp = []
        #global distance:
        for i in range(len(U)):
            ui = U[i]
            uli = ul[i]
            d1 = list(np.sum( np.power(ui-np.array([X[j] for j in range(len(X)) if y[j]==uli]).reshape(-1,D),2), 1))
            d2 = list(np.sum( np.power(ui-np.array([X[j] for j in range(len(X)) if y[j]!=uli]).reshape(-1,D),2), 1))
            d1=sorted(d1)
            d2=sorted(d2)
            s1=0
            s2=0
            for j in range(int(min(len(d1),len(d2))*K)):
                s1 = s1+d1[j]
                s2 = s2+d2[j]
            tmp.append( (U[i],float(s2)/s1,ul[i]) )

        sorted( tmp,key = lambda x:x[1],reverse = True )
        suspect = [(tmp[i][0],tmp[i][2]) for i in range( int(len(tmp)*thres) )  if tmp[i][2]==1]

        if len(suspect) == 0:
            print("not find any suspect at round %d, break\n" % t)
            break
        else:
            print("at %d round, %d samples has been removed\n"%(t, len(suspect) ))
        for i in suspect:
            U.remove(i[0])
        totRemoved = totRemoved+len(suspect)
        X.extend([i[0] for i in suspect])
        y.extend([i[1] for i in suspect])

    print("total removed: %d"%totRemoved)
    return m


def score(yr,yp,n):
    tp=0
    tn=0
    fp=0
    fn=0
    precision=0
    recall = 0
    N=0.0
    P=0.0
    for i in range(len(yr)):
        if yr[i]==0:
            if yp[i]==0:
                tn=tn+1
            else:
                fp = fp+1
            N=N+1
        if yr[i]==1:
            if yp[i]==0:
                fn=fn+1
            else:
                tp=tp+1
            P=P+1
    precision = tp*1.0/(tp+fp)
    recall = tp*1.0/(tp+fn)
    print("tp: %r, fp: %r, tn: %r, fn: %r, n: %r, precision: %r, recall: %r\n"%(tp,fp,tn,fn,n, precision,recall))
    return tp/P,fp/N,tn/N,fn/P,(tp+tn)/float(n),precision,recall

def estimate(model,X,y,U,UL):
    N1 = len(y)
    N2 = len(UL)
    D = len(X[0])
    yp = list(model.predict(np.array(X).reshape(N1,D)))
    up = list(model.predict(np.array(U).reshape(N2,D)))
    
    #input('wait')
    yp.extend(up)
    y.extend(UL)

    tp,fp,tn,fn,p,precision,recall = score(y,yp,N1+N2)
    print("tp: %.4f, tn: %.4f, p: %.4f, precision: %.4f, recall: %.4f\n"%(tp,tn,p,precision,recall))
    return [p,precision,recall]

def main():
    X1 = []
    y1 = []
    X2 = []
    y2 = []
    n = 0
    with open('train_noscale.csv','r') as fin:
        reader = csv.reader(fin)
        firstRow = True
        for row in reader:
            X1.append(list(map(float,row[0:10])))
            y1.append(int(row[-1]))
    with open('test_noscale.csv','r') as fin:
        reader = csv.reader(fin)
        for row in reader:
            X2.append(list(map(float,row[0:10])))
            y2.append(int(row[-1]))
    R=[]
    legends=['p','precision','recall']
    experimentTimes = 10
    for T in range(experimentTimes):
        X,y,U,UL = splitData(X1,y1)
        model = selfTrain(X,y,U,UL)

        X,y,U,UL = splitData(X2,y2)
        result = estimate(model,X,y,U,UL)
        R.append(result)
    R = np.array(R)
    pyplot.figure()
    for j in range(len(legends)):
        pyplot.plot(R[:,j])
    pyplot.legend(legends)
    global uTitle
    pyplot.title(uTitle)
    pyplot.show()

if __name__ == "__main__":
    main()